import random
import time
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, ConfusionMatrixDisplay, precision_score, f1_score, recall_score
import torch
from sklearn.model_selection import train_test_split
from torch import nn, optim
from torch.utils.data import TensorDataset, DataLoader, Dataset
from sklearn.utils import compute_class_weight
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
from torch.nn.utils.rnn import pad_sequence,  pack_padded_sequence

# Set seeds
seed = 69
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)

# Set GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"The device is set to: {device}")

The device is set to: cuda

data_1 = pd.read_json("/kaggle/input/lab22222/train.json").drop(columns='md5hash')
data_2 = pd.read_json("/kaggle/input/lab22222/test.json").drop(columns='md5hash')

previous_length = len(data_1)
data_1 = data_1.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_1)} duplicates \nDataset 1 length: {len(data_1)}")

previous_length = len(data_2)
data_2 = data_2.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_2)} duplicates \nDataset 2 length: {len(data_2)}")

Removed 17875 duplicates 
Dataset 1 length: 12838
Removed 5814 duplicates 
Dataset 2 length: 7349

X = data_1.iloc[:, :-1].values  # Remove label column
X = [x[0] for x in X]
y = data_1.iloc[:, -1].values

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=seed)  # 80% of the data is used for training
X_test = data_2.iloc[:, :-1].values  # Remove label column
X_test = [x[0] for x in X_test]
y_test = data_2.iloc[:, -1].values

X_train_str = [str(x) for x in X_train]
X_val_str = [str(x) for x in X_val]
X_test_str = [str(x) for x in X_test]

vectorizer = CountVectorizer()

# Computing the matrix representing the words contained in each api sequence
document_term_matrix_training = vectorizer.fit_transform(X_train_str)  # The vocabulary is computed on the training set and is used to compute the matrix for the other datasets
document_term_matrix_validation = vectorizer.transform(X_val_str)
document_term_matrix_testing = vectorizer.transform(X_test_str)

# There are 249 columns representing the vocabulary size computed on the training set
print(document_term_matrix_training.shape)

(10270, 249)

# Computing the words contained in the training set vocabulary which are not inside the testing set vocabulary
# The out-of-vocabulary words are ignored by CountVectorizer by default

vectorizer_testing = CountVectorizer()
vectorizer_testing.fit(X_test_str)
out_of_vocabulary = vectorizer.vocabulary_.keys() - vectorizer_testing.vocabulary_.keys()

print(f"There are {len(out_of_vocabulary)} Out-of-vocabulary words from the test set:\n{out_of_vocabulary}")

There are 26 Out-of-vocabulary words from the test set:
{'netusergetlocalgroups', 'sendto', 'ntloadkey', 'ntwritevirtualmemory', 'httpqueryinfoa', 'findfirstfileexa', 'internetqueryoptiona', 'getkeyboardstate', 'listen', 'internetreadfile', 'setinformationjobobject', 'getaddrinfow', 'sendnotifymessagew', 'ntreadvirtualmemory', 'ntqueryfullattributesfile', 'internetopenurlw', 'internetsetstatuscallback', 'setfileinformationbyhandle', 'netusergetinfo', 'ntterminatethread', 'wsasocketw', 'certopensystemstorew', 'cryptdecodeobjectex', 'getusernameexw', 'shutdown', 'certopenstore'}

# The solver is 'saga' because it is fast for large dataset
# The max_iter is 2400 because if it is lower the solver fails to converge (it hasn't reached the global optimum* to within the specified tolerance)
# The class_weight is used to consider the different number of sample for each class (malware and benign)

classifier = LogisticRegression(random_state=seed, solver='saga', max_iter=2400, class_weight='balanced')
classifier.fit(document_term_matrix_training, y_train)

train_predictions = classifier.predict(document_term_matrix_training)
validation_predictions = classifier.predict(document_term_matrix_validation)
test_predictions = classifier.predict(document_term_matrix_testing)

# Evaluating the classifier
train_accuracy = accuracy_score(y_train, train_predictions)
validation_accuracy = accuracy_score(y_val, validation_predictions)
test_accuracy = accuracy_score(y_test, test_predictions)

train_precision = precision_score(y_train, train_predictions, average=None)
validation_precision = precision_score(y_val, validation_predictions, average=None)
test_precision = precision_score(y_test, test_predictions, average=None)

train_recall = recall_score(y_train, train_predictions, average=None)
validation_recall = recall_score(y_val, validation_predictions, average=None)
test_recall = recall_score(y_test, test_predictions, average=None)

train_f1 = f1_score(y_train, train_predictions, average=None)
validation_f1 = f1_score(y_val, validation_predictions, average=None)
test_f1 = f1_score(y_test, test_predictions, average=None)

print(f'Train Accuracy: {train_accuracy:.4f}')
print(f'Train Precision Scores: {train_precision}')
print(f'Train Recall Scores: {train_recall}')
print(f'Train F1 Scores: {train_f1}')

print(f'Validation Accuracy: {validation_accuracy:.4f}')
print(f'Validation Precision Scores: {validation_precision}')
print(f'Validation Recall Scores: {validation_recall}')
print(f'Validation F1 Scores: {validation_f1}')

print(f'Test Accuracy: {test_accuracy:.4f}')
print(f'Test Precision Scores: {test_precision}')
print(f'Test Recall Scores: {test_recall}')
print(f'Test F1 Scores: {test_f1}')

# Generate confusion matrix
ConfusionMatrixDisplay.from_predictions(y_test, test_predictions, cmap="Blues", normalize="true")

Train Accuracy: 0.8665
Train Precision Scores: [0.2281918  0.99590116]
Train Recall Scores: [0.91860465 0.86422764]
Train F1 Scores: [0.36557149 0.92540399]
Validation Accuracy: 0.8520
Validation Precision Scores: [0.17312073 0.99201503]
Validation Recall Scores: [0.8172043  0.85333333]
Validation F1 Scores: [0.28571429 0.91746308]
Test Accuracy: 0.8611
Test Precision Scores: [0.17523168 0.99318403]
Test Recall Scores: [0.832      0.86209325]
Test F1 Scores: [0.289492   0.92300731]

<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x79a69ecbac10>

# Computing mean for each API sequence length for training and testing set
api_calls_training_len = [len(seq) for seq in X_train]
api_calls_testing_len = [len(seq) for seq in X_test]

print(f"Training set sequences length -> Mean: {round(np.mean(api_calls_training_len),2)} - Standard deviation: {round(np.std(api_calls_training_len), 2)}")
print(f"Testing set sequences length  -> Mean: {round(np.mean(api_calls_testing_len), 2)} - Standard deviation: {round(np.std(api_calls_testing_len), 2)}")

Training set sequences length -> Mean: 76.13 - Standard deviation: 8.86
Testing set sequences length  -> Mean: 85.75 - Standard deviation: 8.88

# Assigning sequential IDs to training set
for i in range(0, len(X_train)):
    row = X_train[i]
    for j in range(len(row)):
        row[j] = vectorizer.vocabulary_[row[j].lower()]
    if len(row) < 100:  ## To pad the sequences with less than 100 elements
        for k in range(0, 100 - len(row)):
            row.append(max(vectorizer.vocabulary_.values()) + 1)
    X_train[i] = row

# Assigning sequential IDs to validation set
for i in range(0, len(X_val)):
    row = X_val[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 2
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    if len(row) < 100:  ## To pad the sequences with less than 100 elements
        for k in range(0, 100 - len(row)):
            row.append(max(vectorizer.vocabulary_.values()) + 1)
    X_val[i] = row

# Assigning sequential IDs to testing set
for i in range(0, len(X_test)):
    row = X_test[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 2
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    if len(row) < 100:  ## To pad the sequences with less than 100 elements
        for k in range(0, 100 - len(row)):
            row.append(max(vectorizer.vocabulary_.values()) + 1)
    X_test[i] = row

def training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs, patience=10, min_delta=0.001):
    start_time = time.time()
    train_losses = []
    val_losses = []

    best_val_loss = float('inf')
    epochs_no_improvement = 0

    for epoch in tqdm(range(num_epochs), desc="Training"):  # Train for epochs
        train_loss = 0
        val_loss = 0
        model.train()  # Set model to training mode
        for batch_X, batch_y in train_loader:
            batch_X, batch_y = batch_X.to(device), batch_y.to(device)  # Move batch to GPU
            optimizer.zero_grad()  # Clear previous gradients
            outputs = model(batch_X)  # Forward pass
            loss = criterion(outputs, batch_y)  # Compute loss
            loss.backward()  # Backpropagation
            optimizer.step()  # Update model parameters
            train_loss += loss.item() * batch_X.size(0)
        train_loss /= len(train_dataset)
        train_losses.append(train_loss)  # Store training loss

        # Compute Validation loss during training
        model.eval()  # Set model to evaluation mode
        with torch.no_grad():  # Disable gradient computation for validation
            for batch_X, batch_y in val_loader:
                batch_X, batch_y = batch_X.to(device), batch_y.to(device)  # Move batch to GPU
                val_outputs = model(batch_X)  # Forward pass
                loss = criterion(val_outputs, batch_y)  # Compute validation loss
                val_loss += loss.item() * batch_X.size(0)
            val_loss /= len(val_dataset)
            val_losses.append(val_loss)  # Store validation loss

        # Early stopping
        if best_val_loss - val_loss > min_delta:
            best_val_loss = val_loss
            epochs_no_improvement = 0
            torch.save(model.state_dict(), 'best_model.pth')  # Saving the best model
        else:
            epochs_no_improvement += 1
            if epochs_no_improvement >= patience:
                print(f'Early stopping triggered after {epoch + 1} epochs because there was no improvement over {patience} epochs!')
                print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')
                break  # Stop training

        if (epoch + 1) % 20 == 0:
            print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')

    end_time = time.time()
    elapsed_time = end_time - start_time

    print(f'The function took {elapsed_time:.2f} seconds to execute.')

    # Restore best model weights
    model.load_state_dict(torch.load('best_model.pth', map_location=device))

    # Plot the training and validation loss
    plt.figure(figsize=(10, 5))
    plt.plot(train_losses, label='Train Loss')
    plt.plot(val_losses, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Training and Validation Loss')
    plt.legend()
    plt.grid()
    plt.show()

    return

def testing_model(model, dataloader, device):
    model.eval()  # Set the model to evaluation mode
    all_labels = []
    all_predictions = []

    with torch.no_grad():  # Disable gradient computation for efficiency
        for inputs, labels in dataloader:
            inputs, labels = inputs.to(device), labels.to(device)  # Move batch to GPU
            outputs = model(inputs)
            _, predicted = torch.max(outputs, 1)  # Get the class with the highest score
            all_labels.extend(labels.cpu().numpy())
            all_predictions.extend(predicted.cpu().numpy())

    accuracy = accuracy_score(all_labels, all_predictions) * 100
    precision = precision_score(all_labels, all_predictions, average=None) * 100
    recall = recall_score(all_labels, all_predictions, average=None) * 100
    f1 = f1_score(all_labels, all_predictions, average=None) * 100

    return accuracy, precision, recall, f1, all_labels, all_predictions

def stats(accuracy, precision, recall, f1, description):
    print(f'{description} Accuracy: {accuracy:.4f}')
    print(f'{description} Precision Scores: {precision}')
    print(f'{description} Recall Scores: {recall}')
    print(f'{description} F1 Scores: {f1}\n')

class NonLinearModel(nn.Module):
    def __init__(self, in_size, hid1_size, hid2_size, out_size):
        super(NonLinearModel, self).__init__()
        self.l1 = nn.Linear(in_size, hid1_size)
        self.l2 = nn.Linear(hid1_size, hid2_size)
        self.l3 = nn.Linear(hid2_size, out_size)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.4)
        self.bn1 = nn.BatchNorm1d(hid1_size)
        self.bn2 = nn.BatchNorm1d(hid2_size)

    def forward(self, x):
        x = self.l1(x)
        x = self.relu(x)
        x = self.bn1(x)
        x = self.dropout(x)
        x = self.l2(x)
        x = self.bn2(x)
        x = self.relu(x)
        x = self.dropout(x)
        x = self.l3(x)
        return x

X_train_tensor = torch.tensor(X_train, dtype=torch.float32)
y_train_tensor = torch.tensor(y_train, dtype=torch.long)
X_val_tensor = torch.tensor(X_val, dtype=torch.float32)
y_val_tensor = torch.tensor(y_val, dtype=torch.long)
X_test_tensor = torch.tensor(X_test, dtype=torch.float32)
y_test_tensor = torch.tensor(y_test, dtype=torch.long)

train_dataset = TensorDataset(X_train_tensor, y_train_tensor)
val_dataset = TensorDataset(X_val_tensor, y_val_tensor)
test_dataset = TensorDataset(X_test_tensor, y_test_tensor)

train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)

vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for padding and for out-of-vocabulary
input_size = 100
hidden1_size = 32
hidden2_size = 16
output_size = 2

model = NonLinearModel(input_size, hidden1_size, hidden2_size, 2)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)

optimizer = optim.AdamW(model.parameters(), lr=0.0001)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)

stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')

ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.5752, Val Loss: 0.5488
Epoch 40/100, Train Loss: 0.5371, Val Loss: 0.4975
Early stopping triggered after 58 epochs because there was no improvement over 10 epochs!
Epoch 58/100, Train Loss: 0.5186, Val Loss: 0.4879
The function took 23.14 seconds to execute.

Training Accuracy: 75.4138
Training Precision Scores: [11.72816953 98.55303332]
Training Recall Scores: [74.65116279 75.44715447]
Training F1 Scores: [20.27155036 85.46595292]

Validation Accuracy: 74.9221
Validation Precision Scores: [10.01451379 98.72272485]
Validation Recall Scores: [74.19354839 74.94949495]
Validation F1 Scores: [17.64705882 85.20900322]

Test Accuracy: 75.8334
Test Precision Scores: [ 9.37167199 98.64741364]
Test Recall Scores: [70.4        76.02479222]
Test F1 Scores: [16.54135338 85.87112172]

<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x79a6ca210a90>

class EmbeddingsNonLinearModel(nn.Module):
    def __init__(self, vocab_size, seq_size, embed_size, hid_size, out_size):
        super(EmbeddingsNonLinearModel, self).__init__()

        # 1) embedding layer: vocab_size × d
        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=max(vectorizer.vocabulary_.values()) + 1)

        # 2) first linear layer:
        self.fc1 = nn.Linear(seq_size * embed_size, hid_size) # (seq_len·d) → H
        self.bn1 = nn.BatchNorm1d(hid_size)

        # 3) second linear layer: H → 2
        self.fc2 = nn.Linear(hid_size, out_size)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.5)


    def forward(self, x: torch.LongTensor):
        # a) embedding lookup → (B, seq_len, d) certo, perchè se prima avevo per ogni parola d dim, se qui ho 100 parole avrò 100xd dimensioni
        x = self.embedding(x)

        # b) flatten →
        x = x.view(x.size(0), -1) # (B, seq_len·d) e infatti appatta con primo layer lineare definito sopra, in quanto quello è seq_len * d

        # c) linear + ReLU → (B, H)
        x = self.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.bn1(x)

        # d) final linear layer → (B, 2)
        x = self.fc2(x)
        return x

X_train_tensor = torch.tensor(X_train, dtype=torch.long)
y_train_tensor = torch.tensor(y_train, dtype=torch.long)
X_val_tensor = torch.tensor(X_val, dtype=torch.long)
y_val_tensor = torch.tensor(y_val, dtype=torch.long)
X_test_tensor = torch.tensor(X_test, dtype=torch.long)
y_test_tensor = torch.tensor(y_test, dtype=torch.long)

train_dataset = TensorDataset(X_train_tensor, y_train_tensor)
val_dataset = TensorDataset(X_val_tensor, y_val_tensor)
test_dataset = TensorDataset(X_test_tensor, y_test_tensor)

train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)

vocabulary_size_embedding = len(vectorizer.vocabulary_) + 2  # Add 2 for padding for out-of-vocabulary and padding
sequence_size = 100
embedding_size = 8
hidden_size = 32
output_size_embedding = 2

model = EmbeddingsNonLinearModel(vocabulary_size_embedding, sequence_size, embedding_size, hidden_size, output_size_embedding)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor_embedding = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(class_weight_tensor_embedding)

optimizer = optim.AdamW(model.parameters(), lr=0.0005, weight_decay=0.0001)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)

stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')

ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Early stopping triggered after 17 epochs because there was no improvement over 10 epochs!
Epoch 17/100, Train Loss: 0.1611, Val Loss: 0.3958
The function took 6.53 seconds to execute.

Training Accuracy: 94.1870
Training Precision Scores: [41.67497507 99.87050826]
Training Recall Scores: [97.20930233 94.05487805]
Training F1 Scores: [58.33914864 96.87549066]

Validation Accuracy: 91.9003
Validation Precision Scores: [27.62645914 99.04803116]
Validation Recall Scores: [76.34408602 92.48484848]
Validation F1 Scores: [40.57142857 95.65399081]

Test Accuracy: 92.8290
Test Precision Scores: [29.04689864 99.13277512]
Test Recall Scores: [76.8       93.3934357]
Test F1 Scores: [42.15148189 96.17755857]

<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x79a6a00ed850>

import random
import time
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, ConfusionMatrixDisplay, precision_score, f1_score, recall_score
import torch
from sklearn.model_selection import train_test_split
from torch import nn, optim
from torch.utils.data import TensorDataset, DataLoader, Dataset
from sklearn.utils import compute_class_weight
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
from torch.nn.utils.rnn import pad_sequence,  pack_padded_sequence

# Set seeds
seed = 69
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)

# Set GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"The device is set to: {device}")

The device is set to: cuda

data_1 = pd.read_json("/kaggle/input/lab22222/train.json").drop(columns='md5hash')
data_2 = pd.read_json("/kaggle/input/lab22222/test.json").drop(columns='md5hash')

previous_length = len(data_1)
data_1 = data_1.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_1)} duplicates \nDataset length: {len(data_1)}")

previous_length = len(data_2)
data_2 = data_2.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_2)} duplicates \nDataset length: {len(data_2)}")

X = data_1.iloc[:, :-1].values # Remove label column
X = [x[0] for x in X]
y = data_1.iloc[:, -1].values

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=seed)  # 80% of the data is used for training
X_test = data_2.iloc[:, :-1].values # Remove label column
X_test = [x[0] for x in X_test]
y_test = data_2.iloc[:, -1].values

X_train_str = [str(x) for x in X_train]
X_val_str = [str(x) for x in X_val]
X_test_str = [str(x) for x in X_test]

vectorizer = CountVectorizer()

# Computing the matrix representing the words contained in each api sequence
document_term_matrix_training = vectorizer.fit_transform(X_train_str)  # The vocabulary is computed on the training set and is used to compute the matrix for the other datasets

Removed 17875 duplicates 
Dataset length: 12838
Removed 5814 duplicates 
Dataset length: 7349

# Assigning sequential IDs to training set
for i in range(0, len(X_train)):
    row = X_train[i]
    for j in range(len(row)):
        row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_train[i] = row

# Assigning sequential IDs to validation set
for i in range(0, len(X_val)):
    row = X_val[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 2
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_val[i] = row

# Assigning sequential IDs to testing set
for i in range(0, len(X_test)):
    row = X_test[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 2
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_test[i] = row

class TimeSeriesDataset(Dataset):
    def __init__(self, time_series_list, labels_list):
        """
        Initialize the dataset with sequences and corresponding labels.

        Args:
            time_series_list: List of time series sequences
            labels_list: List of corresponding labels
        """
        super(TimeSeriesDataset, self).__init__()
        self.time_series_list = time_series_list  # List of sequences
        self.labels_list = labels_list            # List of labels

    def __len__(self):
        """Returns the total number of samples in the dataset"""
        return len(self.time_series_list)

    def __getitem__(self, idx):
        """
        Retrieves a single sample and its label.

        Args:
            idx: Index of the sample to retrieve

        Returns:
            tuple: (sequence tensor, label tensor)
        """
        sequence = torch.tensor(self.time_series_list[idx], dtype=torch.float32)
        label = torch.tensor(self.labels_list[idx], dtype=torch.long)
        return sequence, label

def collate_fn(batch):
    """
    Custom collate function for DataLoader to handle variable-length sequences.
    Sorts sequences by length (descending) for efficient RNN processing and pads them.

    Args:
        batch: List of (sequence, label) tuples

    Returns:
        tuple: (padded_sequences, sequence_lengths, sorted_labels)
    """
    sequences, labels = zip(*batch) # Unpack batch
    lengths = torch.tensor([len(seq) for seq in sequences]) # Sequence lengths

    # Sort sequences by length (descending) to optimize RNN processing
    sorted_indices = lengths.argsort(descending=True)
    sequences = [sequences[i] for i in sorted_indices]
    labels = torch.stack([labels[i] for i in sorted_indices])  # Convert to tensor
    lengths = lengths[sorted_indices]  # Sort lengths too

    # Pad sequences to have uniform length in a batch
    padded_sequences = pad_sequence(sequences, batch_first=True, padding_value=max(vectorizer.vocabulary_.values()) + 1)
    return padded_sequences, lengths, labels

def training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs, patience=10, min_delta=0.001):
    start_time = time.time()
    train_losses = []
    val_losses = []

    best_val_loss = float('inf')
    epochs_no_improvement = 0

    # Training loop
    for epoch in tqdm(range(num_epochs), desc="Training"):  # Train for epochs
        train_loss = 0
        val_loss = 0
        model.train()  # Set model to training mode
        for batch_X, lengths, batch_y in train_loader:
            batch_X, lengths, batch_y = batch_X.to(device), lengths.to(device), batch_y.to(device)  # Move batch to GPU
            optimizer.zero_grad()  # Clear previous gradients
            outputs = model(batch_X,lengths)  # Forward pass
            loss = criterion(outputs, batch_y)  # Compute loss
            loss.backward()  # Backpropagation
            optimizer.step()  # Update model parameters
            train_loss += loss.item() * batch_X.size(0)
        train_loss /= len(train_dataset)
        train_losses.append(train_loss)  # Store training loss

        # Compute Validation loss during training
        model.eval()  # Set model to evaluation mode
        with torch.no_grad():  # Disable gradient computation for validation
            for batch_X, lengths, batch_y in val_loader:
                batch_X, lengths, batch_y = batch_X.to(device), lengths.to(device), batch_y.to(device)  # Move batch to GPU
                val_outputs = model(batch_X,lengths)  # Forward pass
                loss = criterion(val_outputs, batch_y)  # Compute validation loss
                val_loss += loss.item() * batch_X.size(0)
            val_loss /= len(val_dataset)
            val_losses.append(val_loss)  # Store validation loss

        # Early stopping
        if best_val_loss - val_loss > min_delta:
            best_val_loss = val_loss
            epochs_no_improvement = 0
            torch.save(model.state_dict(), 'best_model.pth')  # Saving the best model
        else:
            epochs_no_improvement += 1
            if epochs_no_improvement >= patience:
                print(f'Early stopping triggered after {epoch + 1} epochs because no improvements over {patience} subsequent epochs!')
                print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')
                break  # Stop training

        if (epoch + 1) % 20 == 0:
            print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')

    end_time = time.time()
    elapsed_time = end_time - start_time

    print(f'The function took {elapsed_time:.2f} seconds to execute.')

    # Restore best model weights
    model.load_state_dict(torch.load('best_model.pth', map_location=device))

    # Plot the training and validation loss
    plt.figure(figsize=(10, 5))
    plt.plot(train_losses, label='Train Loss')
    plt.plot(val_losses, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Training and Validation Loss')
    plt.legend()
    plt.grid()
    plt.show()

    return

def testing_model(model, dataloader, device):
    model.eval()  # Set the model to evaluation mode
    all_labels = []
    all_predictions = []

    with torch.no_grad():  # Disable gradient computation for efficiency
        for inputs, lengths, labels in dataloader:
            inputs, lengths, labels = inputs.to(device), lengths.to(device), labels.to(device)  # Move batch to GPU
            outputs = model(inputs,lengths)
            _, predicted = torch.max(outputs, 1)  # Get the class with the highest score
            all_labels.extend(labels.cpu().numpy())
            all_predictions.extend(predicted.cpu().numpy())

    accuracy = accuracy_score(all_labels, all_predictions) * 100
    precision = precision_score(all_labels, all_predictions, average=None) * 100
    recall = recall_score(all_labels, all_predictions, average=None) * 100
    f1 = f1_score(all_labels, all_predictions, average=None) * 100

    return accuracy, precision, recall, f1, all_labels, all_predictions

def stats(accuracy, precision, recall, f1, description):
    print(f'{description} Accuracy: {accuracy:.4f}')
    print(f'{description} Precision Scores: {precision}')
    print(f'{description} Recall Scores: {recall}')
    print(f'{description} F1 Scores: {f1}\n')

class MonoDirectionalRNN(nn.Module):
    def __init__(self, vocab_size, embed_size, hid_size, out_size):
        super(MonoDirectionalRNN, self).__init__()

        # 1) embedding layer: vocab_size × d
        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=max(vectorizer.vocabulary_.values()) + 1)

        self.hidden_layer_size = hid_size

        # 2) First linear layer: (seq_len·d) → H
        # batch first is used to specify that the matrix in input has dimension: (batch_size, sequence_length, input_size)
        self.rnn = nn.RNN(input_size=embed_size, hidden_size=hid_size, batch_first=True, num_layers=2, nonlinearity="tanh", dropout=0.5)

        # 3) secondo layer lineare: H → 2
        self.linear = nn.Linear(hid_size, out_size)

    """
     Output of rnn:
         - **rnn_out** contains the output for all time steps.
           It has shape (batch_size, sequence_length, hidden_size).
         - **h** contains the hidden states for all layers at the last time step.A
            It has shape (num_layers, batch_size, hidden_size).
         - if different time-series have different sequence_length, rnn_out can generate errors
         - with h we get batch_size instead of sequence_length
           ==> no problem with different sequence_length

         - We do not return them in the tracking loop to
             - avoid problems with the gradient vanish/exploding
             - use the windows independently (if possible) and shuffle the data
        Returns:
         - predictions: The predicted pollution levels for the input sequence.
         Use h from last hidden node helps with time-series having different lengths
    """
    def forward(self, x: torch.LongTensor, lengths):

        # embedding lookup → (B, seq_len, d) # certo, perchè se prima avevo per ogni parola d dim, se qui ho 100 parole avrò 100xd dimensioni
        x = self.embedding(x.long())

        x = pack_padded_sequence(x, lengths.cpu(), batch_first=True, enforce_sorted=True)   # internally it knows, thanks to collate_fn function, which ones are padded

        rnn_out, h = self.rnn(x)

        predictions = self.linear(h[-1])
        return predictions

train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_sizes = [4,8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320,384,512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_dim=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
        output_size = 2
        
        model = MonoDirectionalRNN(vocabulary_size, embedding_size, hidden_dim, output_size)
        model = model.to(device)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        
        optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
        num_epochs = 100
        training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 8

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.5264, Val Loss: 0.6036
Epoch 40/100, Train Loss: 0.4578, Val Loss: 0.5140
Epoch 60/100, Train Loss: 0.4027, Val Loss: 0.4801
Early stopping triggered after 68 epochs because no improvements over 10 subsequent epochs!
Epoch 68/100, Train Loss: 0.3889, Val Loss: 0.4896
The function took 122.79 seconds to execute.

Training Accuracy: 78.8802
Training Precision Scores: [15.31711209 99.40744558]
Training Recall Scores: [89.30232558 78.42479675]
Training F1 Scores: [26.14913177 87.67823666]

Validation Accuracy: 78.5826
Validation Precision Scores: [12.72430669 99.23273657]
Validation Recall Scores: [83.87096774 78.38383838]
Validation F1 Scores: [22.09631728 87.58465011]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 16

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.5278, Val Loss: 0.5982
Epoch 40/100, Train Loss: 0.4726, Val Loss: 0.5960
Epoch 60/100, Train Loss: 0.4490, Val Loss: 0.5663
Early stopping triggered after 69 epochs because no improvements over 10 subsequent epochs!
Epoch 69/100, Train Loss: 0.4747, Val Loss: 0.5710
The function took 124.98 seconds to execute.

Training Accuracy: 74.6446
Training Precision Scores: [13.10251188 99.39923539]
Training Recall Scores: [89.76744186 73.98373984]
Training F1 Scores: [22.86729858 84.82871126]

Validation Accuracy: 72.8193
Validation Precision Scores: [ 9.93377483 99.00717044]
Validation Recall Scores: [80.64516129 72.52525253]
Validation F1 Scores: [17.68867925 83.72201493]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 32

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.5369, Val Loss: 0.5741
Early stopping triggered after 37 epochs because no improvements over 10 subsequent epochs!
Epoch 37/100, Train Loss: 0.5240, Val Loss: 0.5501
The function took 66.35 seconds to execute.

Training Accuracy: 75.9883
Training Precision Scores: [12.79239766 98.93814707]
Training Recall Scores: [81.39534884 75.75203252]
Training F1 Scores: [22.10991788 85.80637735]

Validation Accuracy: 75.5841
Validation Precision Scores: [10.26785714 98.73417722]
Validation Recall Scores: [74.19354839 75.63636364]
Validation F1 Scores: [18.03921569 85.65545642]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64

# mettere e istanziare model con iperparam migliori Trying with embedding size 32 and hidden dim 64

train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_size=32
hidden_dim=64

vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
output_size = 2

model = MonoDirectionalRNN(vocabulary_size, embedding_size, hidden_dim, output_size)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)

optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.3642, Val Loss: 0.5233
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/100, Train Loss: 0.4786, Val Loss: 0.6104
The function took 45.15 seconds to execute.

Test Accuracy: 81.6029
Test Precision Scores: [12.61872456 98.9106383 ]
Test Recall Scores: [74.4        81.85659952]
Test F1 Scores: [21.57772622 89.57915832]

<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x79a6b8b64c10>

class BiDirectionalRNN(nn.Module):
    def __init__(self, vocab_size, embed_size, hid_size, out_size):
        super(BiDirectionalRNN, self).__init__()

        # 1) embedding layer: vocab_size × d
        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=max(vectorizer.vocabulary_.values()) + 1)

        self.hidden_layer_size = hid_size

        # 2) first liner layer: (seq_len·d) → H
        #batch first is used to specify that the matrix in input has dimension: (batch_size, sequence_length, input_size)
        self.rnn = nn.RNN(input_size=embed_size, hidden_size=hid_size, batch_first=True, num_layers=2, dropout=0.5, bidirectional=True)

        # 3) second linear layer: H → 2
        self.linear = nn.Linear(hid_size * 2, out_size)

    """
    Args:
        - input_seq: The input sequence tensor of shape
         (batch_size, sequence_length, input_size).
     Output of rnn:
         - **rnn_out** contains the output for all time steps.
           It has shape (batch_size, sequence_length, hidden_size).
         - **h** contains the hidden states for all layers at the last time step.
            It has shape (num_layers, batch_size, hidden_size).
         - if different time-series have different sequence_length, rnn_out can generate errors
         - with h we get batch_size instead of sequence_length
           ==> no problem with different sequence_length

         - We do not return them in the tracking loop to
             - avoid problems with the gradient vanish/exploding
             - use the windows independently (if possible) and shuffle the data
        Returns:
         - predictions: The predicted pollution levels for the input sequence.
         Use h from last hidden node helps with time-series having different lengths
    """
    def forward(self, x: torch.LongTensor, lengths):
        # x è B,seq_len

        # embedding lookup → (B, seq_len, d) # certo, perchè se prima avevo per ogni parola d dim, se qui ho 100 parole avrò 100xd dimensioni
        x = self.embedding(x.long())
        x = pack_padded_sequence(x, lengths.cpu(), batch_first=True, enforce_sorted=True)

        rnn_out, h = self.rnn(x)

        # Concatenate the final forward and backward hidden states
        # hidden[-2] contains the last forward hidden state, hidden[-1] contains the last backward hidden state
        hidden_cat = torch.cat((h[-2], h[-1]), dim=1)

        predictions = self.linear(hidden_cat)
        return predictions

train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_sizes = [4,8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320,384,512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_size=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
        output_size = 2

        model = BiDirectionalRNN(vocabulary_size, embedding_size, hidden_size, output_size)
        model = model.to(device)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        
        optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
        num_epochs = 100
        training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.4269, Val Loss: 0.4019
Epoch 40/100, Train Loss: 0.3609, Val Loss: 0.3569
Epoch 60/100, Train Loss: 0.3292, Val Loss: 0.3293
Early stopping triggered after 63 epochs because no improvements over 10 subsequent epochs!
Epoch 63/100, Train Loss: 0.3368, Val Loss: 0.3344
The function took 162.66 seconds to execute.

Training Accuracy: 87.9163
Training Precision Scores: [23.40983607 99.16523728]
Training Recall Scores: [83.02325581 88.1300813 ]
Training F1 Scores: [36.52173913 93.32257197]

Validation Accuracy: 87.7336
Validation Precision Scores: [20.4787234  99.27007299]
Validation Recall Scores: [82.79569892 87.91919192]
Validation F1 Scores: [32.8358209  93.25048211]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.3817, Val Loss: 0.3548
Epoch 40/100, Train Loss: 0.3131, Val Loss: 0.3166
Early stopping triggered after 46 epochs because no improvements over 10 subsequent epochs!
Epoch 46/100, Train Loss: 0.3250, Val Loss: 0.3235
The function took 119.46 seconds to execute.

Training Accuracy: 86.0078
Training Precision Scores: [21.37578169 99.36552697]
Training Recall Scores: [87.44186047 85.94512195]
Training F1 Scores: [34.35358611 92.16936407]

Validation Accuracy: 86.0592
Validation Precision Scores: [18.67612293 99.34731935]
Validation Recall Scores: [84.94623656 86.1010101 ]
Validation F1 Scores: [30.62015504 92.25108225]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.3734, Val Loss: 0.3976
Early stopping triggered after 38 epochs because no improvements over 10 subsequent epochs!
Epoch 38/100, Train Loss: 0.2724, Val Loss: 0.3335
The function took 98.71 seconds to execute.

Training Accuracy: 87.3028
Training Precision Scores: [23.83233533 99.62790698]
Training Recall Scores: [92.55813953 87.07317073]
Training F1 Scores: [37.9047619  92.92841649]

Validation Accuracy: 86.9159
Validation Precision Scores: [18.9258312  99.12723932]
Validation Recall Scores: [79.56989247 87.19191919]
Validation F1 Scores: [30.5785124  92.77730009]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64

# 16 64

train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_size=16
hidden_dim=64

vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
output_size = 2

model = BiDirectionalRNN(vocabulary_size, embedding_size, hidden_dim, output_size)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)

optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.2830, Val Loss: 0.3820
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/100, Train Loss: 0.2298, Val Loss: 0.3709
The function took 57.16 seconds to execute.

Test Accuracy: 88.1344
Test Precision Scores: [19.32938856 99.14759274]
Test Recall Scores: [78.4        88.47725032]
Test F1 Scores: [31.01265823 93.509007  ]

<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7a939d2d6050>

class LSTMClassifier(nn.Module):
    def __init__(self, vocab_size, embed_size, hid_size, out_size):
        super(LSTMClassifier, self).__init__()

        # 1) embedding layer: vocab_size × d
        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=max(vectorizer.vocabulary_.values()) + 1)

        # 2) first linear layer: (seq_len·d) → H
        #batch first is used to specify that the matrix in input has dimension:
        #(batch_size, sequence_length, input_size)
        self.lstm = nn.LSTM(input_size=embed_size, hidden_size=hid_size, batch_first=True, num_layers=2, bidirectional=True, dropout=0.5)

        # 3) second linear layer: H → 2
        self.linear = nn.Linear(hid_size * 2, out_size)

        self.dropout = nn.Dropout(0.5)

    """
    Args:
        - input_seq: The input sequence tensor of shape
         (batch_size, sequence_length, input_size).
     Output of rnn:
         - **rnn_out** contains the output for all time steps.
           It has shape (batch_size, sequence_length, hidden_size).
         - **h** contains the hidden states for all layers at the last time step.
            It has shape (num_layers, batch_size, hidden_size).
         - if different time-series have different sequence_length, rnn_out can generate errors
         - with h we get batch_size instead of sequence_length
           ==> no problem with different sequence_length

         - We do not return them in the tracking loop to
             - avoid problems with the gradient vanish/exploding
             - use the windows independently (if possible) and shuffle the data
        Returns:
         - predictions: The predicted pollution levels for the input sequence.
         Use h from last hidden node helps with time-series having different lengths
    """
    def forward(self, x: torch.LongTensor, lengths):

        # embedding lookup → (B, seq_len, d) # certo, perchè se prima avevo per ogni parola d dim, se qui ho 100 parole avrò 100xd dimensioni
        x = self.embedding(x.long())
        x = self.dropout(x)
        x = pack_padded_sequence(x, lengths.cpu(), batch_first=True, enforce_sorted=True)

        x, (hidden, cell) = self.lstm(x)

        # Concatenate the final forward and backward hidden states
        last_forward = hidden[-2, :, :]  # Last layer's forward direction
        last_backward = hidden[-1, :, :]  # Last layer's backward direction

        # Concatenate the final forward and backward hidden states
        hidden_cat = torch.cat((last_forward, last_backward), dim=1)

        output = self.linear(hidden_cat)

        return output

train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_sizes = [4,8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320,384,512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_size=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
        output_size = 2

        model = LSTMClassifier(vocabulary_size, embedding_size, hidden_size, output_size)
        model = model.to(device)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        
        optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
        num_epochs = 100
        training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.4848, Val Loss: 0.4753
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/100, Train Loss: 0.4369, Val Loss: 0.4301
The function took 90.38 seconds to execute.

Training Accuracy: 78.8121
Training Precision Scores: [14.01483924 98.85262621]
Training Recall Scores: [79.06976744 78.80081301]
Training F1 Scores: [23.80952381 87.69509161]

Validation Accuracy: 79.5561
Validation Precision Scores: [13.13993174 99.19273461]
Validation Recall Scores: [82.79569892 79.43434343]
Validation F1 Scores: [22.68041237 88.22077631]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.4722, Val Loss: 0.4299
Epoch 40/100, Train Loss: 0.3970, Val Loss: 0.3658
Early stopping triggered after 43 epochs because no improvements over 10 subsequent epochs!
Epoch 43/100, Train Loss: 0.4133, Val Loss: 0.3598
The function took 121.20 seconds to execute.

Training Accuracy: 84.7712
Training Precision Scores: [19.45043103 99.1799382 ]
Training Recall Scores: [83.95348837 84.80691057]
Training F1 Scores: [31.58355206 91.4320149 ]

Validation Accuracy: 84.8131
Validation Precision Scores: [17.07317073 99.24421351]
Validation Recall Scores: [82.79569892 84.88888889]
Validation F1 Scores: [28.30882353 91.50696864]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.4546, Val Loss: 0.3881
Epoch 40/100, Train Loss: 0.4080, Val Loss: 0.3317
Early stopping triggered after 59 epochs because no improvements over 10 subsequent epochs!
Epoch 59/100, Train Loss: 0.3264, Val Loss: 0.3321
The function took 174.52 seconds to execute.

Training Accuracy: 89.8734
Training Precision Scores: [26.71755725 99.10714286]
Training Recall Scores: [81.39534884 90.24390244]
Training F1 Scores: [40.22988506 94.46808511]

Validation Accuracy: 89.6028
Validation Precision Scores: [23.31288344 99.24174844]
Validation Recall Scores: [81.72043011 89.8989899 ]
Validation F1 Scores: [36.27684964 94.33962264]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512

#16 and hidden dim 512

train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_size=16
hidden_dim=512

vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
output_size = 2

model = LSTMClassifier(vocabulary_size, embedding_size, hidden_dim, output_size)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)

optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')

Training:   0%|          | 0/100 [00:00<?, ?it/s]

Epoch 20/100, Train Loss: 0.4789, Val Loss: 0.3905
Epoch 40/100, Train Loss: 0.4123, Val Loss: 0.3309
Epoch 60/100, Train Loss: 0.1518, Val Loss: 0.2850
Early stopping triggered after 61 epochs because no improvements over 10 subsequent epochs!
Epoch 61/100, Train Loss: 0.1539, Val Loss: 0.2404
The function took 787.23 seconds to execute.

Test Accuracy: 91.9445
Test Precision Scores: [27.734375   99.43777541]
Test Recall Scores: [85.2        92.18199746]
Test F1 Scores: [41.84675835 95.67251462]

<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7a939e3c8690>

from datetime import time
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import accuracy_score, precision_score, f1_score, recall_score, ConfusionMatrixDisplay
import torch
from sklearn.model_selection import train_test_split
from torch import nn, optim
from tqdm.notebook import tqdm
import time
!pip install torch_geometric
from torch_geometric.data import Data, Dataset
from sklearn.preprocessing import OneHotEncoder
import matplotlib.pyplot as plt
from torch_geometric.nn import GCNConv, SAGEConv, GATConv, global_mean_pool
import torch.nn.functional as F
import torch_geometric
from sklearn.utils.class_weight import compute_class_weight
import networkx as nx
import random

# Set seeds
seed = 69
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)

# Set GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"The device is set to: {device}")

Collecting torch_geometric
  Downloading torch_geometric-2.6.1-py3-none-any.whl.metadata (63 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 63.1/63.1 kB 2.3 MB/s eta 0:00:00
Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (3.12.13)
Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (2025.5.1)
Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (3.1.6)
Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (1.26.4)
Requirement already satisfied: psutil>=5.8.0 in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (7.0.0)
Requirement already satisfied: pyparsing in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (3.0.9)
Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (2.32.4)
Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (4.67.1)
Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (2.6.1)
Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (1.3.2)
Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (25.3.0)
Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (1.7.0)
Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (6.6.3)
Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (0.3.2)
Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (1.20.1)
Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch_geometric) (3.0.2)
Requirement already satisfied: mkl_fft in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (1.3.8)
Requirement already satisfied: mkl_random in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (1.2.4)
Requirement already satisfied: mkl_umath in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (0.1.1)
Requirement already satisfied: mkl in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (2025.2.0)
Requirement already satisfied: tbb4py in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (2022.2.0)
Requirement already satisfied: mkl-service in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (2.4.1)
Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->torch_geometric) (3.4.2)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests->torch_geometric) (3.10)
Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests->torch_geometric) (2.5.0)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests->torch_geometric) (2025.6.15)
Requirement already satisfied: intel-openmp<2026,>=2024 in /usr/local/lib/python3.11/dist-packages (from mkl->numpy->torch_geometric) (2024.2.0)
Requirement already satisfied: tbb==2022.* in /usr/local/lib/python3.11/dist-packages (from mkl->numpy->torch_geometric) (2022.2.0)
Requirement already satisfied: tcmlib==1.* in /usr/local/lib/python3.11/dist-packages (from tbb==2022.*->mkl->numpy->torch_geometric) (1.4.0)
Requirement already satisfied: intel-cmplr-lib-rt in /usr/local/lib/python3.11/dist-packages (from mkl_umath->numpy->torch_geometric) (2024.2.0)
Requirement already satisfied: intel-cmplr-lib-ur==2024.2.0 in /usr/local/lib/python3.11/dist-packages (from intel-openmp<2026,>=2024->mkl->numpy->torch_geometric) (2024.2.0)
Downloading torch_geometric-2.6.1-py3-none-any.whl (1.1 MB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.1/1.1 MB 29.4 MB/s eta 0:00:0000:01
Installing collected packages: torch_geometric
Successfully installed torch_geometric-2.6.1
The device is set to: cuda

def seq_to_edge_lists(seq):
    """
    Dato seq = [c0, c1, …, cN-1], costruisce:
      sources = [m(c0), m(c1), …, m(cN-2)]
      dests   = [m(c1), m(c2), …, m(cN-1)]
    dove m(x) è l'indice interno assegnato in ordine di prima apparizione.

    Esempio:
      seq = [3,7,1,3,1,3,2]
      mapping = {3:0, 7:1, 1:2, 2:3}
      sources  = [0,1,2,0,2,0]
      dests    = [1,2,0,2,0,3]
    """
    mapping = {}     # comando_originale -> nuovo_indice
    next_id = 0

    # 1) Crea la mappatura comando->indice
    for cmd in seq:
        if cmd not in mapping:
            mapping[cmd] = next_id
            next_id += 1

    # 2) Costruisci le liste di archi (consecutive pairs)
    sources = []
    dests   = []
    for u, v in zip(seq, seq[1:]):  # zip(seq, seq[1:]) è equivalente a fare for i in range(len(seq) - 1) e prendere u = seq[i] e v = seq[i + 1]
        sources.append(mapping[u])
        dests.append(mapping[v])

    return sources, dests

class GraphDataset(Dataset):
    def __init__(self, data, labels, embedds):
        """
        Initialize dataset by pre-processing all graphs into PyG `Data` format.

        Args:
            data (list of np.array): List of sequences.
            labels (list of int): List of graph labels.
            embeds embedding created outside.
        """
        super().__init__()
        self.graphs = []  # Store processed graphs

        # Create embedding layer with the given vocabulary size and embedding size
        self.embedding_layer = embedds

        for i in range(len(data)):
            # Convert adjacency matrix to edge index
            edge_index = seq_to_edge_lists(data[i])  # shape = (2, E)
            edge_index = torch.tensor(edge_index, dtype=torch.long)

            # Convert node features to embeddings
            x = torch.tensor(data[i], dtype=torch.long)  # Node feature indices
            x = self.embedding_layer(x)  # Convert indices to embeddings
            # Ensure `y` is a single integer label
            #y = torch.tensor(labels[i].nonzero()[0], dtype=torch.long).unsqueeze(0)  # Class index of the graph
            # labels[i].nonzero()[0] to encode the label from a one-hot-encoding representation into a numerical version
            y = labels[i].nonzero()[0].clone().detach().to(dtype=torch.long).unsqueeze(0)


            # Create Data object and store it
            self.graphs.append(Data(x=x, edge_index=edge_index, y=y))

    def len(self):
        return len(self.graphs)

    def get(self, idx):
        return self.graphs[idx]  # Return a pre-processed graph

data_1 = pd.read_json("/kaggle/input/lab22222/train.json").drop(columns='md5hash')
data_2 = pd.read_json("/kaggle/input/lab22222/test.json").drop(columns='md5hash')

previous_length = len(data_1)
data_1 = data_1.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_1)} duplicates \nDataset length: {len(data_1)}")

previous_length = len(data_2)
data_2 = data_2.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_2)} duplicates \nDataset length: {len(data_2)}")

X = data_1.iloc[:, :-1].values # Remove label column
X = [x[0] for x in X]
y = data_1.iloc[:, -1].values
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=seed)  # 80% of the data is used for training
X_test = data_2.iloc[:, :-1].values # Remove label column
X_test = [x[0] for x in X_test]
y_test = data_2.iloc[:, -1].values

X_train_str = [str(x) for x in X_train]
vectorizer = CountVectorizer()

# Computing the matrix representing the words contained in each api sequence
document_term_matrix_training = vectorizer.fit_transform(X_train_str)  # The vocabulary is computed on the training set and is used to compute the matrix for the other datasets

# Assigning sequential IDs to training set
for i in range(0, len(X_train)):
    row = X_train[i]
    for j in range(len(row)):
        row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_train[i] = row

# Assigning sequential IDs to validation set
for i in range(0, len(X_val)):
    row = X_val[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 1
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_val[i] = row

# Assigning sequential IDs to testing set
for i in range(0, len(X_test)):
    row = X_test[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 1
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_test[i] = row

def stats(accuracy, precision, recall, f1, description):
    print(f'{description} Accuracy: {accuracy:.4f}')
    print(f'{description} Precision Scores: {precision}')
    print(f'{description} Recall Scores: {recall}')
    print(f'{description} F1 Scores: {f1}\n')

Removed 17875 duplicates 
Dataset length: 12838
Removed 5814 duplicates 
Dataset length: 7349

# Reshape the labels to ensure compatibility with OneHotEncoder
y_train_original = y_train
y_val_original = y_val
y_test_original = y_test
y_train_reshaped = np.array(y_train).reshape(-1, 1).astype('long')
y_val_reshaped = np.array(y_val).reshape(-1, 1).astype('long')
y_test_reshaped = np.array(y_test).reshape(-1, 1).astype('long')

# Initialize the OneHotEncoder
encoder = OneHotEncoder(sparse_output=False)

# Fit and transform the labels
y_train = torch.tensor(encoder.fit_transform(y_train_reshaped)).to(torch.long)
y_val = torch.tensor(encoder.transform(y_val_reshaped)).to(torch.long)
y_test = torch.tensor(encoder.transform(y_test_reshaped)).to(torch.long)

# Create embedding layer with the given vocabulary size and embedding size
embedding_size = 8 # initially 8 but then we modify it
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)

def plot_graph_simple(data):
    edge_index = data.edge_index.numpy()

    # Create directed graph
    G = nx.DiGraph()

    # Calculate unique nodes strictly from edge_index
    unique_nodes = set(np.concatenate(edge_index))
    for node in unique_nodes:
        G.add_node(node)  # Add only nodes present in edges

    # Add edges to the graph
    for i in range(edge_index.shape[1]):
        source = edge_index[0, i]
        target = edge_index[1, i]
        G.add_edge(source, target)

    # Print graph info
    print(f"NetworkX graph has {G.number_of_nodes()} nodes and {G.number_of_edges()} edges")

    # Plot with different layout options
    plt.figure(figsize=(10, 8))
    pos = nx.kamada_kawai_layout(G)  # Use kamada_kawai_layout for better structure

    # Draw with high visibility settings
    nx.draw(G, pos,
            with_labels=True,
            node_color='skyblue',
            node_size=700,
            font_size=10,
            font_weight='bold',
            arrowsize=20,
            width=2,
            edge_color='red',
            arrows=True)

    plt.title(f"Graph Visualization - Label: {data.y.item()}")
    plt.tight_layout()
    plt.show()

# Re-run the plot
plot_graph_simple(X_train_graph[4])

NetworkX graph has 26 nodes and 54 edges

/tmp/ipykernel_36/2669503328.py:38: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
  plt.tight_layout()

def training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, num_epochs, criterion, patience=10, min_delta=0.001):
    start_time = time.time()
    train_losses = []
    val_losses = []

    best_val_loss = float('inf')
    epochs_no_improvement = 0

    # Training loop
    for epoch in tqdm(range(num_epochs)):  # Train for epochs
        train_loss = 0
        val_loss = 0
        model.train()  # Set model to training mode
        for batch in train_loader:
            batch = batch.to(device)  # Move batch to GPU
            optimizer.zero_grad()  # Clear previous gradients
            outputs = model(batch)  # Forward pass
            loss = criterion(outputs, batch.y.squeeze())  # Compute loss
            loss.backward(retain_graph=True)  # Backpropagation
            optimizer.step()  # Update model parameters
            train_loss += loss.item() * batch.size(0)
        train_loss /= len(train_dataset)
        train_losses.append(train_loss)  # Store training loss

        # Compute Validation loss during training
        model.eval()  # Set model to evaluation mode
        with torch.no_grad():  # Disable gradient computation for validation
            for batch in val_loader:
                batch = batch.to(device)  # Move batch to GPU
                val_outputs = model(batch)  # Forward pass
                loss = criterion(val_outputs, batch.y.squeeze())  # Compute validation loss. .squeeze() is used to reduce from 2D to 1D vector
                val_loss += loss.item() * batch.size(0)
            val_loss /= len(val_dataset)
            val_losses.append(val_loss)  # Store validation loss

        # Early stopping
        if best_val_loss - val_loss > min_delta:
            best_val_loss = val_loss
            epochs_no_improvement = 0
            torch.save(model.state_dict(), 'best_model.pth')  # Saving the best model
        else:
            epochs_no_improvement += 1
            if epochs_no_improvement >= patience:
                print(f'Early stopping triggered after {epoch + 1} epochs because no improvements over {patience} subsequent epochs!')
                print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')
                break  # Stop training

        if (epoch + 1) % 5 == 0:
            print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')

    end_time = time.time()
    elapsed_time = end_time - start_time

    print(f'The function took {elapsed_time:.2f} seconds to execute.')

    # Restore best model weights
    model.load_state_dict(torch.load('best_model.pth', map_location=device))

    # Plot the training and validation loss
    plt.figure(figsize=(10, 5))
    plt.plot(train_losses, label='Train Loss')
    plt.plot(val_losses, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Training and Validation Loss')
    plt.legend()
    plt.grid()
    plt.show()

    return

def testing_model(model, dataloader, device):
    model.eval()  # Set the model to evaluation mode
    all_labels = []
    all_predictions = []

    with torch.no_grad():  # Disable gradient computation for efficiency
        for batch in dataloader:
            batch = batch.to(device)  # Move batch to GPU
            outputs = model(batch)
            _, predicted = torch.max(outputs, 1)  # Get the class with highest score
            all_labels.extend(batch.y.squeeze().cpu().numpy())
            all_predictions.extend(predicted.cpu().numpy())

    accuracy = accuracy_score(all_labels, all_predictions) * 100
    precision = precision_score(all_labels, all_predictions, average=None) * 100
    recall = recall_score(all_labels, all_predictions, average=None) * 100
    f1 = f1_score(all_labels, all_predictions, average=None) * 100

    return accuracy, precision, recall, f1, all_labels, all_predictions

def testing_model_with_threshold(model, dataloader, device, threshold=0.5):
    model.eval()
    all_labels = []
    all_predictions = []
    with torch.no_grad():
        for batch in dataloader:
            batch = batch.to(device)
            outputs = model(batch)
            probabilities = F.softmax(outputs, dim=1)  # Convert logits to probabilities
            predicted = (probabilities[:, 0] > threshold).long()  # 1 if p_class0 > threshold, else 0
            predicted_classes = 1 - predicted  # If predicted=1, class=0; if predicted=0, class=1
            all_labels.extend(batch.y.squeeze().cpu().numpy())
            all_predictions.extend(predicted_classes.cpu().numpy())
    accuracy = accuracy_score(all_labels, all_predictions) * 100
    precision = precision_score(all_labels, all_predictions, average=None) * 100
    recall = recall_score(all_labels, all_predictions, average=None) * 100
    f1 = f1_score(all_labels, all_predictions, average=None) * 100
    return accuracy, precision, recall, f1, all_labels, all_predictions

class GCN(torch.nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(GCN, self).__init__()
        self.conv1 = GCNConv(input_dim, hidden_dim)
        self.conv2 = GCNConv(hidden_dim, hidden_dim)
        self.fc = torch.nn.Linear(hidden_dim, output_dim)
        self.bn = torch.nn.BatchNorm1d(hidden_dim)
        self.dropout = torch.nn.Dropout(0.4)

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch
        x = F.relu(self.conv1(x, edge_index))
        x = self.dropout(x)
        x = self.bn(x)
        x = F.relu(self.conv2(x, edge_index))
        x = global_mean_pool(x, batch)  # Aggregate node features from Node to Graph features
        x = self.fc(x)
        return x

class GraphSAGE(torch.nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(GraphSAGE, self).__init__()
        self.conv1 = SAGEConv(input_dim, hidden_dim)
        self.conv2 = SAGEConv(hidden_dim, hidden_dim)
        self.fc = torch.nn.Linear(hidden_dim, output_dim)
        self.dropout = torch.nn.Dropout(0.4)

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch
        x = F.relu(self.conv1(x, edge_index))
        x = self.dropout(x)
        x = F.relu(self.conv2(x, edge_index))
        x = global_mean_pool(x, batch) # Aggregate node features. From Node to Graph Features
        x = self.fc(x)
        return x

class GAT(torch.nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim, heads=4):
        super(GAT, self).__init__()
        self.conv1 = GATConv(input_dim, hidden_dim, heads=heads, concat=True)
        self.conv2 = GATConv(hidden_dim * heads, hidden_dim, heads=1, concat=False)
        self.fc = torch.nn.Linear(hidden_dim, output_dim)

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch
        x = F.elu(self.conv1(x, edge_index))
        x = F.elu(self.conv2(x, edge_index))
        x = global_mean_pool(x, batch) # Aggregate node features. From Node to Graph Features
        x = self.fc(x)
        return x

embedding_sizes = [8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320, 384, 512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_dim=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        # Create embedding layer with the given vocabulary size and embedding size
        vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
        embeddings = nn.Embedding(vocabulary_size, embedding_size)
        X_train_graph = GraphDataset(X_train, y_train, embeddings)
        X_val_graph = GraphDataset(X_val, y_val, embeddings)
        X_test_graph = GraphDataset(X_test, y_test, embeddings)
        
        # Create DataLoader for batching
        # NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
        batch_size = 64
        train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
        val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
        test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        num_epochs = 50
        model = GCN(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
        optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
        training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        #test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
        #stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
        
        # Test with different thresholds
        for threshold in [0.5, 0.6, 0.7, 0.8, 0.9]:
            print(f"Threshold: {threshold}")
            val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model_with_threshold(model,val_loader,device,threshold=threshold)
            stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
            print()

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 8

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 46.1322, Val Loss: 44.1200
Epoch 10/50, Train Loss: 43.3972, Val Loss: 43.6996
Epoch 15/50, Train Loss: 41.7698, Val Loss: 43.1021
Epoch 20/50, Train Loss: 43.0420, Val Loss: 42.7623
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/50, Train Loss: 43.3233, Val Loss: 41.6739
The function took 30.87 seconds to execute.

Training Accuracy: 72.8432
Training Precision Scores: [11.05975569 98.68802652]
Training Recall Scores: [77.90697674 72.62195122]
Training F1 Scores: [19.36976005 83.67191616]

Validation Accuracy: 70.8333
Validation Precision Scores: [ 9.2039801  98.92290249]
Validation Recall Scores: [79.56989247 70.50505051]
Validation F1 Scores: [16.49944259 82.33073838]

Threshold: 0.5
Validation Accuracy: 70.8333
Validation Precision Scores: [ 9.2039801  98.92290249]
Validation Recall Scores: [79.56989247 70.50505051]
Validation F1 Scores: [16.49944259 82.33073838]


Threshold: 0.6
Validation Accuracy: 79.5561
Validation Precision Scores: [11.5658363  98.60418744]
Validation Recall Scores: [69.89247312 79.91919192]
Validation F1 Scores: [19.84732824 88.28386521]


Threshold: 0.7
Validation Accuracy: 86.9159
Validation Precision Scores: [13.06990881 97.76686021]
Validation Recall Scores: [46.23655914 88.44444444]
Validation F1 Scores: [20.37914692 92.87229529]


Threshold: 0.8
Validation Accuracy: 92.5623
Validation Precision Scores: [15.         97.03459638]
Validation Recall Scores: [22.58064516 95.19191919]
Validation F1 Scores: [18.02575107 96.10442586]


Threshold: 0.9
Validation Accuracy: 95.5607
Validation Precision Scores: [20.         96.60481642]
Validation Recall Scores: [ 7.52688172 98.86868687]
Validation F1 Scores: [10.9375     97.72364217]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 16

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 42.9639, Val Loss: 42.7165
Epoch 10/50, Train Loss: 39.4247, Val Loss: 38.9916
Epoch 15/50, Train Loss: 38.9545, Val Loss: 42.3849
Epoch 20/50, Train Loss: 38.6360, Val Loss: 37.6971
Epoch 25/50, Train Loss: 37.5386, Val Loss: 36.4397
Epoch 30/50, Train Loss: 37.9233, Val Loss: 37.0997
Epoch 35/50, Train Loss: 37.3246, Val Loss: 35.8904
Epoch 40/50, Train Loss: 36.9472, Val Loss: 36.9899
Early stopping triggered after 43 epochs because no improvements over 10 subsequent epochs!
Epoch 43/50, Train Loss: 36.8017, Val Loss: 38.0919
The function took 60.72 seconds to execute.

Training Accuracy: 82.4343
Training Precision Scores: [15.92261905 98.67942816]
Training Recall Scores: [74.65116279 82.77439024]
Training F1 Scores: [26.24693377 90.02984415]

Validation Accuracy: 82.2040
Validation Precision Scores: [14.453125   99.07587549]
Validation Recall Scores: [79.56989247 82.3030303 ]
Validation F1 Scores: [24.46280992 89.91392629]

Threshold: 0.5
Validation Accuracy: 82.2040
Validation Precision Scores: [14.453125   99.07587549]
Validation Recall Scores: [79.56989247 82.3030303 ]
Validation F1 Scores: [24.46280992 89.91392629]


Threshold: 0.6
Validation Accuracy: 88.0062
Validation Precision Scores: [15.87301587 98.09143364]
Validation Recall Scores: [53.76344086 89.29292929]
Validation F1 Scores: [24.50980392 93.4856176 ]


Threshold: 0.7
Validation Accuracy: 91.9003
Validation Precision Scores: [17.14285714 97.36732135]
Validation Recall Scores: [32.25806452 94.14141414]
Validation F1 Scores: [22.3880597  95.72719803]


Threshold: 0.8
Validation Accuracy: 95.0545
Validation Precision Scores: [20.68965517 96.77290837]
Validation Recall Scores: [12.90322581 98.14141414]
Validation F1 Scores: [15.89403974 97.45235707]


Threshold: 0.9
Validation Accuracy: 95.7555
Validation Precision Scores: [23.33333333 96.61150512]
Validation Recall Scores: [ 7.52688172 99.07070707]
Validation F1 Scores: [11.38211382 97.8256533 ]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 32

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 40.6945, Val Loss: 37.1508
Epoch 10/50, Train Loss: 37.8760, Val Loss: 34.2618
Epoch 15/50, Train Loss: 36.9977, Val Loss: 32.8120
Epoch 20/50, Train Loss: 35.8666, Val Loss: 37.7323
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/50, Train Loss: 36.9255, Val Loss: 34.5277
The function took 34.81 seconds to execute.

Training Accuracy: 82.9503
Training Precision Scores: [16.11082606 98.60593679]
Training Recall Scores: [73.02325581 83.38414634]
Training F1 Scores: [26.39764607 90.35846044]

Validation Accuracy: 83.5670
Validation Precision Scores: [15.07430998 98.95088221]
Validation Recall Scores: [76.34408602 83.83838384]
Validation F1 Scores: [25.17730496 90.76990376]

Threshold: 0.5
Validation Accuracy: 83.5670
Validation Precision Scores: [15.07430998 98.95088221]
Validation Recall Scores: [76.34408602 83.83838384]
Validation F1 Scores: [25.17730496 90.76990376]


Threshold: 0.6
Validation Accuracy: 89.4860
Validation Precision Scores: [19.58762887 98.41897233]
Validation Recall Scores: [61.29032258 90.54545455]
Validation F1 Scores: [29.6875     94.31818182]


Threshold: 0.7
Validation Accuracy: 92.9128
Validation Precision Scores: [23.35329341 97.75093711]
Validation Recall Scores: [41.93548387 94.82828283]
Validation F1 Scores: [30.         96.26743232]


Threshold: 0.8
Validation Accuracy: 95.0156
Validation Precision Scores: [26.02739726 97.03406814]
Validation Recall Scores: [20.43010753 97.81818182]
Validation F1 Scores: [22.89156627 97.42454728]


Threshold: 0.9
Validation Accuracy: 96.1838
Validation Precision Scores: [40.         96.73613842]
Validation Recall Scores: [10.75268817 99.39393939]
Validation F1 Scores: [16.94915254 98.04703069]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64

# 16 and hidden dim 128 threshold 0.9

embedding_size=16
hidden_dim=128
threshold = 0.9 

# Create embedding layer with the given vocabulary size and embedding size
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)

# Create DataLoader for batching
# NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
batch_size = 64
train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
num_epochs = 50
model = GCN(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model_with_threshold(model,val_loader,device,threshold=threshold)

stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 37.3670, Val Loss: 41.0201
Epoch 10/50, Train Loss: 33.9726, Val Loss: 34.1704
Epoch 15/50, Train Loss: 31.5596, Val Loss: 29.4678
Epoch 20/50, Train Loss: 30.3961, Val Loss: 26.6791
Epoch 25/50, Train Loss: 27.9115, Val Loss: 27.4083
Epoch 30/50, Train Loss: 26.0020, Val Loss: 25.5574
Epoch 35/50, Train Loss: 26.8470, Val Loss: 27.6204
Early stopping triggered after 40 epochs because no improvements over 10 subsequent epochs!
Epoch 40/50, Train Loss: 25.3462, Val Loss: 26.0847
The function took 57.13 seconds to execute.

Test Accuracy: 96.4953
Test Precision Scores: [52.83018868 97.41550696]
Test Recall Scores: [30.10752688 98.98989899]
Test F1 Scores: [38.35616438 98.19639279]

<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7a93bae125d0>

embedding_sizes = [8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320, 384, 512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_dim=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        # Create embedding layer with the given vocabulary size and embedding size
        vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
        embeddings = nn.Embedding(vocabulary_size, embedding_size)
        X_train_graph = GraphDataset(X_train, y_train, embeddings)
        X_val_graph = GraphDataset(X_val, y_val, embeddings)
        X_test_graph = GraphDataset(X_test, y_test, embeddings)
        
        # Create DataLoader for batching
        # NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
        batch_size = 64
        train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
        val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
        test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        num_epochs = 50
        model = GraphSAGE(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
        optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
        training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        #test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
        #stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
        
        # Test with different thresholds
        for threshold in [0.5, 0.6, 0.7, 0.8, 0.9]:
            print(f"Threshold: {threshold}")
            val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model_with_threshold(model,val_loader,device,threshold=threshold)
            stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
            print()

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 8

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 42.4182, Val Loss: 40.8873
Epoch 10/50, Train Loss: 40.8287, Val Loss: 38.8620
Epoch 15/50, Train Loss: 39.1520, Val Loss: 40.7064
Epoch 20/50, Train Loss: 38.8658, Val Loss: 34.9887
Epoch 25/50, Train Loss: 37.7685, Val Loss: 34.5356
Epoch 30/50, Train Loss: 36.5703, Val Loss: 33.0668
Epoch 35/50, Train Loss: 36.5097, Val Loss: 33.4247
Epoch 40/50, Train Loss: 35.6928, Val Loss: 38.1752
Epoch 45/50, Train Loss: 35.5721, Val Loss: 36.1797
Early stopping triggered after 46 epochs because no improvements over 10 subsequent epochs!
Epoch 46/50, Train Loss: 35.4310, Val Loss: 35.4654
The function took 60.77 seconds to execute.

Training Accuracy: 85.0828
Training Precision Scores: [17.31909846 98.39235788]
Training Recall Scores: [67.90697674 85.83333333]
Training F1 Scores: [27.59924386 91.68475901]

Validation Accuracy: 85.5919
Validation Precision Scores: [15.97051597 98.70430356]
Validation Recall Scores: [69.89247312 86.18181818]
Validation F1 Scores: [26.         92.01898188]

Threshold: 0.5
Validation Accuracy: 85.5919
Validation Precision Scores: [15.97051597 98.70430356]
Validation Recall Scores: [69.89247312 86.18181818]
Validation F1 Scores: [26.         92.01898188]


Threshold: 0.6
Validation Accuracy: 90.6153
Validation Precision Scores: [21.53846154 98.39688042]
Validation Recall Scores: [60.21505376 91.75757576]
Validation F1 Scores: [31.72804533 94.96132135]


Threshold: 0.7
Validation Accuracy: 94.1589
Validation Precision Scores: [30.87248322 98.05704837]
Validation Recall Scores: [49.46236559 95.83838384]
Validation F1 Scores: [38.01652893 96.93502248]


Threshold: 0.8
Validation Accuracy: 96.4564
Validation Precision Scores: [52.38095238 97.18923199]
Validation Recall Scores: [23.65591398 99.19191919]
Validation F1 Scores: [32.59259259 98.18036393]


Threshold: 0.9
Validation Accuracy: 96.6511
Validation Precision Scores: [73.33333333 96.78809244]
Validation Recall Scores: [11.82795699 99.83838384]
Validation F1 Scores: [20.37037037 98.28957836]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 16

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 37.8303, Val Loss: 35.5702
Epoch 10/50, Train Loss: 34.6404, Val Loss: 31.7704
Epoch 15/50, Train Loss: 34.0041, Val Loss: 30.7726
Epoch 20/50, Train Loss: 33.7227, Val Loss: 28.8863
Epoch 25/50, Train Loss: 32.1090, Val Loss: 29.3035
Epoch 30/50, Train Loss: 31.4762, Val Loss: 28.6804
Epoch 35/50, Train Loss: 32.0113, Val Loss: 29.1743
Early stopping triggered after 36 epochs because no improvements over 10 subsequent epochs!
Epoch 36/50, Train Loss: 30.5809, Val Loss: 28.0976
The function took 46.85 seconds to execute.

Training Accuracy: 85.6475
Training Precision Scores: [19.65116279 98.92397661]
Training Recall Scores: [78.60465116 85.95528455]
Training F1 Scores: [31.44186047 91.98477433]

Validation Accuracy: 85.9034
Validation Precision Scores: [17.89976134 99.16240112]
Validation Recall Scores: [80.64516129 86.1010101 ]
Validation F1 Scores: [29.296875   92.17128028]

Threshold: 0.5
Validation Accuracy: 85.9034
Validation Precision Scores: [17.89976134 99.16240112]
Validation Recall Scores: [80.64516129 86.1010101 ]
Validation F1 Scores: [29.296875   92.17128028]


Threshold: 0.6
Validation Accuracy: 93.1075
Validation Precision Scores: [29.61165049 98.64521592]
Validation Recall Scores: [65.59139785 94.14141414]
Validation F1 Scores: [40.80267559 96.34070705]


Threshold: 0.7
Validation Accuracy: 95.4050
Validation Precision Scores: [39.31623932 98.08241534]
Validation Recall Scores: [49.46236559 97.13131313]
Validation F1 Scores: [43.80952381 97.6045473 ]


Threshold: 0.8
Validation Accuracy: 96.4953
Validation Precision Scores: [52.72727273 97.45324314]
Validation Recall Scores: [31.1827957  98.94949495]
Validation F1 Scores: [39.18918919 98.19566961]


Threshold: 0.9
Validation Accuracy: 96.9237
Validation Precision Scores: [79.16666667 97.09119497]
Validation Recall Scores: [20.43010753 99.7979798 ]
Validation F1 Scores: [32.47863248 98.42598127]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 32

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 37.2210, Val Loss: 33.0439
Epoch 10/50, Train Loss: 32.3825, Val Loss: 30.1625
Epoch 15/50, Train Loss: 30.3157, Val Loss: 25.6824
Epoch 20/50, Train Loss: 28.3542, Val Loss: 26.6730
Epoch 25/50, Train Loss: 27.3200, Val Loss: 23.7659
Epoch 30/50, Train Loss: 25.7724, Val Loss: 25.0714
Epoch 35/50, Train Loss: 25.4951, Val Loss: 22.5241
Epoch 40/50, Train Loss: 25.2975, Val Loss: 24.5805
Epoch 45/50, Train Loss: 25.4734, Val Loss: 22.4624
Epoch 50/50, Train Loss: 25.4526, Val Loss: 23.5583
The function took 66.54 seconds to execute.

Training Accuracy: 86.6310
Training Precision Scores: [22.57126236 99.50882938]
Training Recall Scores: [90.23255814 86.47357724]
Training F1 Scores: [36.10981852 92.53439182]

Validation Accuracy: 86.6433
Validation Precision Scores: [19.80676329 99.48932219]
Validation Recall Scores: [88.17204301 86.58585859]
Validation F1 Scores: [32.34714004 92.59019227]

Threshold: 0.5
Validation Accuracy: 86.6433
Validation Precision Scores: [19.80676329 99.48932219]
Validation Recall Scores: [88.17204301 86.58585859]
Validation F1 Scores: [32.34714004 92.59019227]


Threshold: 0.6
Validation Accuracy: 91.5498
Validation Precision Scores: [27.53623188 99.2582897 ]
Validation Recall Scores: [81.72043011 91.91919192]
Validation F1 Scores: [41.19241192 95.44787078]


Threshold: 0.7
Validation Accuracy: 93.9252
Validation Precision Scores: [34.01015228 98.90341628]
Validation Recall Scores: [72.04301075 94.74747475]
Validation F1 Scores: [46.20689655 96.78085019]


Threshold: 0.8
Validation Accuracy: 95.8333
Validation Precision Scores: [44.6969697  98.60426929]
Validation Recall Scores: [63.44086022 97.05050505]
Validation F1 Scores: [52.44444444 97.82121767]


Threshold: 0.9
Validation Accuracy: 97.0405
Validation Precision Scores: [61.64383562 98.0761523 ]
Validation Recall Scores: [48.38709677 98.86868687]
Validation F1 Scores: [54.21686747 98.47082495]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64

# 16 and hidden dim 256 threshold 0.9

embedding_size=16
hidden_dim=256
threshold = 0.9 

# Create embedding layer with the given vocabulary size and embedding size
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)

# Create DataLoader for batching
# NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
batch_size = 64
train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
num_epochs = 50
model = GraphSAGE(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model_with_threshold(model,val_loader,device,threshold=threshold)

stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 29.8540, Val Loss: 32.2394
Epoch 10/50, Train Loss: 23.8979, Val Loss: 23.4067
Epoch 15/50, Train Loss: 22.0149, Val Loss: 25.1712
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/50, Train Loss: 20.2248, Val Loss: 23.6084
The function took 29.33 seconds to execute.

Test Accuracy: 96.5732
Test Precision Scores: [53.01204819 98.02816901]
Test Recall Scores: [47.31182796 98.42424242]
Test F1 Scores: [50.         98.22580645]

<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7a93abb7ff10>

embedding_sizes = [8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320, 384, 512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_dim=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        # Create embedding layer with the given vocabulary size and embedding size
        vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
        embeddings = nn.Embedding(vocabulary_size, embedding_size)
        X_train_graph = GraphDataset(X_train, y_train, embeddings)
        X_val_graph = GraphDataset(X_val, y_val, embeddings)
        X_test_graph = GraphDataset(X_test, y_test, embeddings)
        
        # Create DataLoader for batching
        # NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
        batch_size = 64
        train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
        val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
        test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        num_epochs = 50
        model = GAT(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
        optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
        training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        #test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
        #stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
        
        # Test with different thresholds
        for threshold in [0.5, 0.6, 0.7, 0.8, 0.9]:
            print(f"Threshold: {threshold}")
            val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model_with_threshold(model,val_loader,device,threshold=threshold)
            stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
            print()

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 8

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 39.0102, Val Loss: 38.1614
Epoch 10/50, Train Loss: 33.1951, Val Loss: 32.8341
Epoch 15/50, Train Loss: 30.4372, Val Loss: 30.8448
Epoch 20/50, Train Loss: 30.3905, Val Loss: 31.0182
Epoch 25/50, Train Loss: 28.0026, Val Loss: 26.8883
Epoch 30/50, Train Loss: 27.4088, Val Loss: 27.7825
Early stopping triggered after 33 epochs because no improvements over 10 subsequent epochs!
Epoch 33/50, Train Loss: 26.3089, Val Loss: 29.9954
The function took 56.98 seconds to execute.

Training Accuracy: 86.1831
Training Precision Scores: [20.7223209  99.06770773]
Training Recall Scores: [81.39534884 86.39227642]
Training F1 Scores: [33.03445021 92.29683513]

Validation Accuracy: 86.7212
Validation Precision Scores: [19.45812808 99.35245143]
Validation Recall Scores: [84.94623656 86.78787879]
Validation F1 Scores: [31.66332665 92.6461074 ]

Threshold: 0.5
Validation Accuracy: 86.7212
Validation Precision Scores: [19.45812808 99.35245143]
Validation Recall Scores: [84.94623656 86.78787879]
Validation F1 Scores: [31.66332665 92.6461074 ]


Threshold: 0.6
Validation Accuracy: 90.9657
Validation Precision Scores: [25.26690391 99.0380411 ]
Validation Recall Scores: [76.34408602 91.51515152]
Validation F1 Scores: [37.96791444 95.12809744]


Threshold: 0.7
Validation Accuracy: 94.4315
Validation Precision Scores: [35.46511628 98.66444073]
Validation Recall Scores: [65.59139785 95.51515152]
Validation F1 Scores: [46.03773585 97.06425785]


Threshold: 0.8
Validation Accuracy: 96.3785
Validation Precision Scores: [50.         98.10177706]
Validation Recall Scores: [49.46236559 98.14141414]
Validation F1 Scores: [49.72972973 98.1215916 ]


Threshold: 0.9
Validation Accuracy: 97.0405
Validation Precision Scores: [68.08510638 97.58032527]
Validation Recall Scores: [34.40860215 99.39393939]
Validation F1 Scores: [45.71428571 98.47878303]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 16

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 38.8971, Val Loss: 35.2171
Epoch 10/50, Train Loss: 33.2826, Val Loss: 31.8862
Epoch 15/50, Train Loss: 31.6364, Val Loss: 27.9429
Epoch 20/50, Train Loss: 31.4714, Val Loss: 29.5137
Epoch 25/50, Train Loss: 28.5403, Val Loss: 24.5040
Epoch 30/50, Train Loss: 29.9110, Val Loss: 28.7493
Early stopping triggered after 35 epochs because no improvements over 10 subsequent epochs!
Epoch 35/50, Train Loss: 27.2175, Val Loss: 24.9689
The function took 60.12 seconds to execute.

Training Accuracy: 88.0526
Training Precision Scores: [23.27297116 99.05456202]
Training Recall Scores: [80.69767442 88.37398374]
Training F1 Scores: [36.12701718 93.40995757]

Validation Accuracy: 89.0187
Validation Precision Scores: [22.12389381 99.19246299]
Validation Recall Scores: [80.64516129 89.33333333]
Validation F1 Scores: [34.72222222 94.00510204]

Threshold: 0.5
Validation Accuracy: 89.0187
Validation Precision Scores: [22.12389381 99.19246299]
Validation Recall Scores: [80.64516129 89.33333333]
Validation F1 Scores: [34.72222222 94.00510204]


Threshold: 0.6
Validation Accuracy: 92.2118
Validation Precision Scores: [28.51405622 99.05131522]
Validation Recall Scores: [76.34408602 92.80808081]
Validation F1 Scores: [41.52046784 95.82811848]


Threshold: 0.7
Validation Accuracy: 94.0421
Validation Precision Scores: [32.75862069 98.4962406 ]
Validation Recall Scores: [61.29032258 95.27272727]
Validation F1 Scores: [42.69662921 96.85767098]


Threshold: 0.8
Validation Accuracy: 96.1838
Validation Precision Scores: [47.36842105 98.05903761]
Validation Recall Scores: [48.38709677 97.97979798]
Validation F1 Scores: [47.87234043 98.01940178]


Threshold: 0.9
Validation Accuracy: 96.8069
Validation Precision Scores: [61.2244898  97.49900754]
Validation Recall Scores: [32.25806452 99.23232323]
Validation F1 Scores: [42.25352113 98.35802964]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 32

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 41.5580, Val Loss: 57.4460
Epoch 10/50, Train Loss: 35.8319, Val Loss: 33.9315
Epoch 15/50, Train Loss: 33.2896, Val Loss: 34.8361
Epoch 20/50, Train Loss: 31.0224, Val Loss: 40.6379
Epoch 25/50, Train Loss: 28.6341, Val Loss: 31.0588
Epoch 30/50, Train Loss: 28.1938, Val Loss: 29.2019
Epoch 35/50, Train Loss: 25.9330, Val Loss: 27.9621
Epoch 40/50, Train Loss: 26.5195, Val Loss: 28.8038
Epoch 45/50, Train Loss: 25.4159, Val Loss: 26.9866
Epoch 50/50, Train Loss: 25.6434, Val Loss: 30.0298
The function took 86.10 seconds to execute.

Training Accuracy: 85.7936
Training Precision Scores: [20.51575931 99.15542522]
Training Recall Scores: [83.25581395 85.90447154]
Training F1 Scores: [32.91954023 92.05554043]

Validation Accuracy: 85.6308
Validation Precision Scores: [18.05555556 99.29775281]
Validation Recall Scores: [83.87096774 85.6969697 ]
Validation F1 Scores: [29.71428571 91.99739753]

Threshold: 0.5
Validation Accuracy: 85.6308
Validation Precision Scores: [18.05555556 99.29775281]
Validation Recall Scores: [83.87096774 85.6969697 ]
Validation F1 Scores: [29.71428571 91.99739753]


Threshold: 0.6
Validation Accuracy: 90.9657
Validation Precision Scores: [25.78397213 99.167032  ]
Validation Recall Scores: [79.56989247 91.39393939]
Validation F1 Scores: [38.94736842 95.12195122]


Threshold: 0.7
Validation Accuracy: 93.4190
Validation Precision Scores: [31.55339806 98.81456393]
Validation Recall Scores: [69.89247312 94.3030303 ]
Validation F1 Scores: [43.47826087 96.50609882]


Threshold: 0.8
Validation Accuracy: 95.5997
Validation Precision Scores: [41.93548387 98.32242226]
Validation Recall Scores: [55.91397849 97.09090909]
Validation F1 Scores: [47.92626728 97.70278512]


Threshold: 0.9
Validation Accuracy: 96.7679
Validation Precision Scores: [58.33333333 97.68740032]
Validation Recall Scores: [37.6344086  98.98989899]
Validation F1 Scores: [45.75163399 98.33433674]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64

#16 and hidden dim 16  threhsol 0.8

embedding_size=16
hidden_dim=16
threshold = 0.8

# Create embedding layer with the given vocabulary size and embedding size
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)

# Create DataLoader for batching
# NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
batch_size = 64
train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
num_epochs = 50
model = GAT(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model_with_threshold(model,val_loader,device,threshold=threshold)

stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 34.6077, Val Loss: 33.0638
Epoch 10/50, Train Loss: 29.7320, Val Loss: 27.8456
Epoch 15/50, Train Loss: 26.9065, Val Loss: 25.7897
Epoch 20/50, Train Loss: 23.3077, Val Loss: 26.8553
Epoch 25/50, Train Loss: 24.6751, Val Loss: 27.1542
Epoch 30/50, Train Loss: 21.1759, Val Loss: 26.9506
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/50, Train Loss: 20.9676, Val Loss: 27.2604
The function took 50.99 seconds to execute.

Test Accuracy: 94.4315
Test Precision Scores: [35.63218391 98.70509607]
Test Recall Scores: [66.66666667 95.47474747]
Test F1 Scores: [46.44194757 97.06305196]

<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7d3f7e3ea7d0>

# 16 and hidden dim 128 threshold 0.9

device = 'cpu'
embedding_size=16
hidden_dim=128
threshold = 0.9 

# Create embedding layer with the given vocabulary size and embedding size
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)

# Create DataLoader for batching
# NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
batch_size = 64
train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
num_epochs = 50
model = GCN(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)

train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)

stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')

# Test with threshold
print(f"Threshold: {threshold}")
test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model_with_threshold(model, test_loader, device, threshold=threshold)
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, f'Test {threshold}')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
print()

  0%|          | 0/50 [00:00<?, ?it/s]

Epoch 5/50, Train Loss: 36.8246, Val Loss: 35.8298
Epoch 10/50, Train Loss: 33.8961, Val Loss: 32.5482
Epoch 15/50, Train Loss: 31.0544, Val Loss: 34.1158
Epoch 20/50, Train Loss: 29.4902, Val Loss: 29.2766
Epoch 25/50, Train Loss: 27.9879, Val Loss: 30.2824
Epoch 30/50, Train Loss: 27.1075, Val Loss: 26.0054
Epoch 35/50, Train Loss: 26.2199, Val Loss: 27.2533
Epoch 40/50, Train Loss: 25.4207, Val Loss: 25.0337
Epoch 45/50, Train Loss: 25.4130, Val Loss: 26.8479
Early stopping triggered after 50 epochs because no improvements over 10 subsequent epochs!
Epoch 50/50, Train Loss: 24.1975, Val Loss: 28.1010
The function took 204.92 seconds to execute.

Training Accuracy: 87.8968
Training Precision Scores: [24.41787288 99.51618477]
Training Recall Scores: [90.23255814 87.79471545]
Training F1 Scores: [38.43486875 93.28869931]

Validation Accuracy: 87.1106
Validation Precision Scores: [20.10050251 99.40092166]
Validation Recall Scores: [86.02150538 87.15151515]
Validation F1 Scores: [32.58655804 92.87405813]

Test Accuracy: 87.7398
Test Precision Scores: [19.14691943 99.23736892]
Test Recall Scores: [80.8        87.98422313]
Test F1 Scores: [30.95785441 93.27260509]

Threshold: 0.9
Test 0.9 Accuracy: 97.1425
Test 0.9 Precision Scores: [64.49275362 97.76729996]
Test 0.9 Recall Scores: [35.6        99.30976194]
Test 0.9 F1 Scores: [45.87628866 98.53249476]

Preprocessing¶

Task 1: BoW Approaches¶

Applying vectorizer¶

Test set out-of-vocabulary words¶

Fit a classifier¶

Logistic regression¶

Task 2: FFNN¶

Using sequential IDs¶

Using embeddings¶

Task 3: RNN¶

Preprocessing¶

Monodirectional¶

Bidirectional¶

LSTM¶

Task 4: GNN¶

Preprocessing¶

Modifying the labels using one hot encoding¶

GNN¶

Simple GCN¶

Graph SAGE¶

GAT¶

Simple GCN on CPU¶